import scrapy

from xiaoshuo.items import XiaoshuoItem


class xiaoshuo(scrapy.Spider):
    name = 'rain'
    start_urls = [
        'https://www.biquge.info/wanjiexiaoshuo/1'
    ]
    custom_settings = {
        "USER_AGENT": "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36",
    }

    def parse(self, response):
        urls = response.xpath('//*[@id="main"]/div[1]/ul/li/span[2]/a/@href').extract()
        for url in urls:
            yield scrapy.Request(url, callback=self.details)

        next_page = response.xpath("//*[@id='pagelink']/a[@class='next']/@href").extract()[0]
        if next_page is not None:
            next_page = response.urljoin(next_page)
            yield scrapy.Request(next_page, callback=self.parse)

    def details(self, response):
        item = XiaoshuoItem()
        name = response.xpath("//*[@id='info']/h1/text()").extract()
        jianjie = response.xpath("//*[@id='intro']/p/text()").extract()[0]
        url = response.request.url
        if len(jianjie) == 0:
            jianjie = "null"
            
        item['name'] = name
        item['url'] = url
        item['jianjie'] = str(jianjie)
        yield item
